{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/component_examples/named_entity_recognition_NER/NLU_explain_clinical_doc_generic_pipeline.ipynb)\n",
        "\n",
        "#Explain Clinical Document Generic\n",
        "\n",
        "This pipeline is designed to:\n",
        "\n",
        "- extract clinical/medical entities\n",
        "- assign assertion status to the extracted entities\n",
        "- establish relations between the extracted entities\n",
        "\n",
        "from clinical texts. In this pipeline, 4 NER models, one assertion model, and one relation extraction model were used to achieve those tasks."
      ],
      "metadata": {
        "id": "jJ4POSNF8qzD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0mxspMbm8fcE"
      },
      "outputs": [],
      "source": [
        "! pip install nlu pyspark==3.1.2"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install johnsnowlabs"
      ],
      "metadata": {
        "id": "kEyCNAVO9PdX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json, os\n",
        "from google.colab import files\n",
        "\n",
        "if 'spark_jsl.json' not in os.listdir():\n",
        "  license_keys = files.upload()\n",
        "  os.rename(list(license_keys.keys())[0], 'spark_jsl.json')\n",
        "\n",
        "with open('spark_jsl.json') as f:\n",
        "    license_keys = json.load(f)\n",
        "\n",
        "# Defining license key-value pairs as local variables\n",
        "locals().update(license_keys)\n",
        "os.environ.update(license_keys)"
      ],
      "metadata": {
        "id": "a1HT1V_N9THF"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Installing pyspark and spark-nlp\n",
        "! pip install --upgrade -q pyspark==3.1.2 spark-nlp==$PUBLIC_VERSION\n",
        "\n",
        "# Installing NLU\n",
        "! pip install --upgrade --q nlu --no-dependencies\n",
        "\n",
        "# Installing Spark NLP Healthcare\n",
        "! pip install --upgrade -q spark-nlp-jsl==$JSL_VERSION  --extra-index-url https://pypi.johnsnowlabs.com/$SECRET\n",
        "\n",
        "# Installing Spark NLP Display Library for visualization\n",
        "! pip install -q spark-nlp-display"
      ],
      "metadata": {
        "id": "s1fprtHN9TFO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import os\n",
        "\n",
        "import sparknlp\n",
        "import sparknlp_jsl\n",
        "import nlu\n",
        "\n",
        "from sparknlp.base import *\n",
        "from sparknlp.annotator import *\n",
        "from sparknlp_jsl.annotator import *\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import functions as F\n",
        "from pyspark.ml import Pipeline,PipelineModel\n",
        "\n",
        "import pandas as pd\n",
        "pd.set_option('display.max_colwidth', 200)\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "params = {\"spark.driver.memory\":\"16G\",\n",
        "          \"spark.kryoserializer.buffer.max\":\"2000M\",\n",
        "          \"spark.driver.maxResultSize\":\"2000M\"}\n",
        "\n",
        "print(\"Spark NLP Version :\", sparknlp.version())\n",
        "print(\"Spark NLP_JSL Version :\", sparknlp_jsl.version())\n",
        "\n",
        "spark = sparknlp_jsl.start(license_keys['SECRET'],params=params)\n",
        "\n",
        "spark"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "drDXkUeq9TBZ",
        "outputId": "61e33d7d-e789-4bee-c114-dcdb596aeb88"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark NLP Version : 5.3.0\n",
            "Spark NLP_JSL Version : 5.3.0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7cdaef842920>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://654d17a7d193:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Spark NLP Licensed</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = nlu.load(\"en.explain_doc.clinical_generic.pipeline\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qZ_4Fz559S_V",
        "outputId": "e25c43dd-bdce-49ad-d5c1-4188f7e2e2f4"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning::Spark Session already created, some configs may not take.\n",
            "Warning::Spark Session already created, some configs may not take.\n",
            "explain_clinical_doc_generic download started this may take some time.\n",
            "Approx size to download 1.7 GB\n",
            "[OK!]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = [\"\"\"Patient with severe fever and sore throat.\n",
        "He shows no stomach pain. He maintained on an epidural and PCA for pain control.\n",
        "After CT, lung tumor located at the right lower lobe. Father with Alzheimer.\"\"\"]"
      ],
      "metadata": {
        "id": "62fOREz--k3I"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = model.predict(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xFyyGMFN9S9W",
        "outputId": "ba48da6f-25e8-4fa4-9c25-6cacaebabab7"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning::Spark Session already created, some configs may not take.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "KuCeqmwN9S66",
        "outputId": "8ee8b116-5d3f-46a9-a307-ddfaf5ead09f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                     assertion  \\\n",
              "0  [Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]   \n",
              "0  [Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]   \n",
              "0  [Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]   \n",
              "0  [Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]   \n",
              "\n",
              "                                                                                                                                                                                                  document  \\\n",
              "0  Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...   \n",
              "0  Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...   \n",
              "0  Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...   \n",
              "0  Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...   \n",
              "\n",
              "                                                                            entities_clinical_ner_chunk  \\\n",
              "0  [severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]   \n",
              "0  [severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]   \n",
              "0  [severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]   \n",
              "0  [severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]   \n",
              "\n",
              "                                                      entities_clinical_ner_chunk_class  \\\n",
              "0  [PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]   \n",
              "0  [PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]   \n",
              "0  [PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]   \n",
              "0  [PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]   \n",
              "\n",
              "                                          entities_clinical_ner_chunk_confidence  \\\n",
              "0  [0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]   \n",
              "0  [0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]   \n",
              "0  [0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]   \n",
              "0  [0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]   \n",
              "\n",
              "  entities_clinical_ner_chunk_origin_chunk  \\\n",
              "0              [0, 1, 2, 3, 4, 5, 6, 7, 8]   \n",
              "0              [0, 1, 2, 3, 4, 5, 6, 7, 8]   \n",
              "0              [0, 1, 2, 3, 4, 5, 6, 7, 8]   \n",
              "0              [0, 1, 2, 3, 4, 5, 6, 7, 8]   \n",
              "\n",
              "  entities_clinical_ner_chunk_origin_sentence  \\\n",
              "0                 [0, 0, 1, 2, 2, 2, 3, 3, 4]   \n",
              "0                 [0, 0, 1, 2, 2, 2, 3, 3, 4]   \n",
              "0                 [0, 0, 1, 2, 2, 2, 3, 3, 4]   \n",
              "0                 [0, 0, 1, 2, 2, 2, 3, 3, 4]   \n",
              "\n",
              "                                                     entities_jsl_ner_chunk  \\\n",
              "0  [fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]   \n",
              "0  [fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]   \n",
              "0  [fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]   \n",
              "0  [fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]   \n",
              "\n",
              "                                           entities_jsl_ner_chunk_class  \\\n",
              "0  [TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]   \n",
              "0  [TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]   \n",
              "0  [TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]   \n",
              "0  [TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]   \n",
              "\n",
              "                                           entities_jsl_ner_chunk_confidence  \\\n",
              "0  [0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]   \n",
              "0  [0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]   \n",
              "0  [0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]   \n",
              "0  [0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]   \n",
              "\n",
              "   ... relations_entity1_class relations_entity1_end relations_entity2  \\\n",
              "0  ...                 PROBLEM                    24       sore throat   \n",
              "0  ...               TREATMENT                    96               PCA   \n",
              "0  ...               TREATMENT                   104              pain   \n",
              "0  ...                    TEST                   131        lung tumor   \n",
              "\n",
              "  relations_entity2_begin relations_entity2_class relations_entity2_end  \\\n",
              "0                      30                 PROBLEM                    40   \n",
              "0                     102               TREATMENT                   104   \n",
              "0                     110                 PROBLEM                   113   \n",
              "0                     134                 PROBLEM                   143   \n",
              "\n",
              "  relations_origin_sentence  \\\n",
              "0                         0   \n",
              "0                         2   \n",
              "0                         2   \n",
              "0                         3   \n",
              "\n",
              "                                                                                                                                                                                               sentence_dl  \\\n",
              "0  [Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...   \n",
              "0  [Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...   \n",
              "0  [Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...   \n",
              "0  [Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...   \n",
              "\n",
              "                                                                                                                                                                                      unlabeled_dependency  \\\n",
              "0  [ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...   \n",
              "0  [ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...   \n",
              "0  [ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...   \n",
              "0  [ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...   \n",
              "\n",
              "                                                                                                                                                                                 word_embedding_embeddings  \n",
              "0  [[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...  \n",
              "0  [[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...  \n",
              "0  [[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...  \n",
              "0  [[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...  \n",
              "\n",
              "[4 rows x 34 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7ffa1d09-6028-4fad-bf40-17776aa7ab23\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>assertion</th>\n",
              "      <th>document</th>\n",
              "      <th>entities_clinical_ner_chunk</th>\n",
              "      <th>entities_clinical_ner_chunk_class</th>\n",
              "      <th>entities_clinical_ner_chunk_confidence</th>\n",
              "      <th>entities_clinical_ner_chunk_origin_chunk</th>\n",
              "      <th>entities_clinical_ner_chunk_origin_sentence</th>\n",
              "      <th>entities_jsl_ner_chunk</th>\n",
              "      <th>entities_jsl_ner_chunk_class</th>\n",
              "      <th>entities_jsl_ner_chunk_confidence</th>\n",
              "      <th>...</th>\n",
              "      <th>relations_entity1_class</th>\n",
              "      <th>relations_entity1_end</th>\n",
              "      <th>relations_entity2</th>\n",
              "      <th>relations_entity2_begin</th>\n",
              "      <th>relations_entity2_class</th>\n",
              "      <th>relations_entity2_end</th>\n",
              "      <th>relations_origin_sentence</th>\n",
              "      <th>sentence_dl</th>\n",
              "      <th>unlabeled_dependency</th>\n",
              "      <th>word_embedding_embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]</td>\n",
              "      <td>Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...</td>\n",
              "      <td>[severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]</td>\n",
              "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8]</td>\n",
              "      <td>[0, 0, 1, 2, 2, 2, 3, 3, 4]</td>\n",
              "      <td>[fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]</td>\n",
              "      <td>...</td>\n",
              "      <td>PROBLEM</td>\n",
              "      <td>24</td>\n",
              "      <td>sore throat</td>\n",
              "      <td>30</td>\n",
              "      <td>PROBLEM</td>\n",
              "      <td>40</td>\n",
              "      <td>0</td>\n",
              "      <td>[Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...</td>\n",
              "      <td>[ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...</td>\n",
              "      <td>[[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]</td>\n",
              "      <td>Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...</td>\n",
              "      <td>[severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]</td>\n",
              "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8]</td>\n",
              "      <td>[0, 0, 1, 2, 2, 2, 3, 3, 4]</td>\n",
              "      <td>[fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]</td>\n",
              "      <td>...</td>\n",
              "      <td>TREATMENT</td>\n",
              "      <td>96</td>\n",
              "      <td>PCA</td>\n",
              "      <td>102</td>\n",
              "      <td>TREATMENT</td>\n",
              "      <td>104</td>\n",
              "      <td>2</td>\n",
              "      <td>[Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...</td>\n",
              "      <td>[ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...</td>\n",
              "      <td>[[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]</td>\n",
              "      <td>Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...</td>\n",
              "      <td>[severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]</td>\n",
              "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8]</td>\n",
              "      <td>[0, 0, 1, 2, 2, 2, 3, 3, 4]</td>\n",
              "      <td>[fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]</td>\n",
              "      <td>...</td>\n",
              "      <td>TREATMENT</td>\n",
              "      <td>104</td>\n",
              "      <td>pain</td>\n",
              "      <td>110</td>\n",
              "      <td>PROBLEM</td>\n",
              "      <td>113</td>\n",
              "      <td>2</td>\n",
              "      <td>[Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...</td>\n",
              "      <td>[ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...</td>\n",
              "      <td>[[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[Present, Present, Absent, Past, Past, Hypothetical, Past, Present, Family]</td>\n",
              "      <td>Patient with severe fever and sore throat.\\nHe shows no stomach pain. He maintained on an epidural and PCA for pain control.\\nAfter CT, lung tumor located at the right lower lobe. Father with Alzh...</td>\n",
              "      <td>[severe fever, sore throat, stomach pain, an epidural, PCA, pain control, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[PROBLEM, PROBLEM, PROBLEM, TREATMENT, TREATMENT, TREATMENT, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.902, 0.98815, 0.9933, 0.96599996, 0.9813, 0.73165, 0.9933, 0.9019, 0.9912]</td>\n",
              "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8]</td>\n",
              "      <td>[0, 0, 1, 2, 2, 2, 3, 3, 4]</td>\n",
              "      <td>[fever, sore throat, stomach pain, PCA, pain, CT, lung tumor, Alzheimer]</td>\n",
              "      <td>[TEST, PROBLEM, PROBLEM, TREATMENT, PROBLEM, TEST, PROBLEM, PROBLEM]</td>\n",
              "      <td>[0.9943, 0.69635, 0.82834995, 0.8087, 0.9875, 0.9934, 0.87944996, 0.9796]</td>\n",
              "      <td>...</td>\n",
              "      <td>TEST</td>\n",
              "      <td>131</td>\n",
              "      <td>lung tumor</td>\n",
              "      <td>134</td>\n",
              "      <td>PROBLEM</td>\n",
              "      <td>143</td>\n",
              "      <td>3</td>\n",
              "      <td>[Patient with severe fever and sore throat., He shows no stomach pain., He maintained on an epidural and PCA for pain control., After CT, lung tumor located at the right lower lobe., Father with A...</td>\n",
              "      <td>[ROOT, fever, fever, Patient, sore, throat, fever, Patient, shows, ROOT, stomach, pain, shows, shows, maintained, ROOT, epidural, epidural, maintained, PCA, epidural, control, control, PCA, mainta...</td>\n",
              "      <td>[[-0.08453157544136047, 0.20632991194725037, -0.05394839495420456, -0.364096075296402, -0.7453896999359131, -0.45486748218536377, 0.1968030333518982, -0.7794908285140991, -0.08118873834609985, 0.2...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4 rows × 34 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ffa1d09-6028-4fad-bf40-17776aa7ab23')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7ffa1d09-6028-4fad-bf40-17776aa7ab23 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7ffa1d09-6028-4fad-bf40-17776aa7ab23');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3875f87e-35f8-4163-a823-5d754f72f9bf\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3875f87e-35f8-4163-a823-5d754f72f9bf')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3875f87e-35f8-4163-a823-5d754f72f9bf button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_e3a5e21a-1feb-473f-83f4-29b6d0b2b6dc\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_e3a5e21a-1feb-473f-83f4-29b6d0b2b6dc button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ShKt275l9S40"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "E4t7g6jS9S2W"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}